# coding:utf8
# 百度知道长尾关键词挖掘
import sys
import importlib, sys
import random

import CONST
import time

importlib.reload(sys)
import requests, re
from requests.packages.urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
headers = {

    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0"
}


def get_html(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
        'Cookie': 'HMACCOUNT_BFESS=C5EE59B2468B08DE; BIDUPSID=107CB9984E0DC532CD9271AD15668FF4; PSTM=1652002030; BAIDUID=107CB9984E0DC532CD9271AD15668FF4:SL=0:NR=10:FG=1; ZFY=E1ZzK7qg:BgeTV70pi6NWBs6AIOmpWP0LIpRAkxv5aOc:C; __bid_n=183893bbdfb680567c4207; BAIDU_WISE_UID=wapp_1665037363577_152; BAIDUID_BFESS=107CB9984E0DC532CD9271AD15668FF4:SL=0:NR=10:FG=1; newlogin=1; BDUSS=0lMZU4yQklicXFVRWdWNXZoQVltQWtKUTljbnBvajNMUFNmUEt0c2RVdEloWU5qSVFBQUFBJCQAAAAAAQAAAAEAAABBZ6Eh0-rSubHVw8VpAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEj4W2NI-Ftjb; BDUSS_BFESS=0lMZU4yQklicXFVRWdWNXZoQVltQWtKUTljbnBvajNMUFNmUEt0c2RVdEloWU5qSVFBQUFBJCQAAAAAAQAAAAEAAABBZ6Eh0-rSubHVw8VpAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEj4W2NI-Ftjb; RT="z=1&dm=baidu.com&si=756005d0-3ff5-4a2b-8626-ef6e40aac2da&ss=l9t82won&sl=0&tt=0&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=6a6&ul=46xoi&hd=46xqj"; ZD_ENTRY=bing; ab_sr=1.0.1_YzMwMWExNzE1YzBiOGIyYTc5NDRkOGRmMWQyMDY2MjlmMjVlODM2YzI0ZjcxYTQyNzM3ZWIxMTczZTIyNzhjZTVjY2VjMzM1YTk0NjgxZTIyZTBiYTAyNmZmODliMDAwMmFmZjg4MjdkOTJjZmFlMjc2ZGY1NGY4MzljMDM4MGQxM2Y4Y2VjNGJlNWI5ZDJkNjc4ZTVmNjliNDUzOTkzMjZiYTgxNzJmZDhiYTNiZWNmYjIwZjhhNTAwZWQzMmRj'
    }
    while 1:
        try:
            time.sleep(random.randint(1,20))
            r = requests.get(url, verify=False, headers=headers)
            r.raise_for_status()  # 如果次参数不是200，则产生异常，进入异常处理语句，否则程序继续往下走
            # print(r.text)
            r.encoding = "gbk"
            html = r.text
            return html
        except Exception as e:
            print(e)
            return ''


def get_info(url):
    html = get_html(url)
    html = re.sub('<em>', '', html)
    html = re.sub('</em>', '', html)
    titles = re.findall('target="_blank" class="ti">([\s\S]*?)</a>\s+</dt>', html)
    return titles

def get_long_tail_key(seed, num=CONST.NUM_LONG_TAIL):
    """
    百度知道长尾关键词挖掘工具
    :param seed: 种子关键词
    :param num: 需要挖掘的长尾关键词数量
    :return: [’长尾关键词‘, ]
    """
    base_url = 'https://zhidao.baidu.com/search?lm=0&rn=10&pn=%s&fr=search&ie=gbk&dyTabStr=null&word=%s'

    url1 = base_url % (0, seed)
    html1 = get_html(url1)
    id1 = re.findall(u'<span class="f-lighter lh-22">共(.*?)条结果</span>', html1)[0]
    id1 = re.sub(',', '', id1)
    id2 = int(id1)

    long_tail = []
    for id in range(0, id2):  # 有多少页遍历多少页
        if len(long_tail) >= num:
            break
        num1 = id * 10
        url = base_url % (num1, seed)
        # print('正在挖掘第', id + 1, '页的关键词')
        long_tail.extend(get_info(url))
    return long_tail



if __name__ == "__main__":
   re = get_long_tail_key('新闻', 200)
   print(re)
